{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notebook written by [Zhedong Zheng](https://github.com/zhedongzheng)\n",
    "\n",
    "<img src=\"transformer.png\" width=\"250\">\n",
    "\n",
    "[Attention Is All You Need](https://arxiv.org/abs/1706.03762)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bunch import Bunch\n",
    "from collections import Counter\n",
    "\n",
    "import json\n",
    "import numpy as np\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "args = Bunch({\n",
    "    'source_max_len': 10,\n",
    "    'target_max_len': 10,\n",
    "    'min_freq': 50,\n",
    "    'hidden_units': 128,\n",
    "    'num_blocks': 2,\n",
    "    'num_heads': 8,\n",
    "    'num_heads': 8,\n",
    "    'dropout_rate': 0.1,\n",
    "    'batch_size': 64,\n",
    "    'position_encoding': 'non_param',\n",
    "    'activation': 'relu',\n",
    "    'tied_proj_weight': True,\n",
    "    'tied_embedding': False,\n",
    "    'label_smoothing': False,\n",
    "    'lr_decay_strategy': 'exp',\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DataLoader:\n",
    "    def __init__(self, source_path, target_path):\n",
    "        self.source_words = self.read_data(source_path)\n",
    "        self.target_words = self.read_data(target_path)\n",
    "\n",
    "        self.source_word2idx = self.build_index(self.source_words)\n",
    "        self.target_word2idx = self.build_index(self.target_words, is_target=True)\n",
    "\n",
    "    \n",
    "    def read_data(self, path):\n",
    "        with open(path, 'r', encoding='utf-8') as f:\n",
    "            return f.read()\n",
    "\n",
    "\n",
    "    def build_index(self, data, is_target=False):\n",
    "        chars = [char for line in data.split('\\n') for char in line]\n",
    "        chars = [char for char, freq in Counter(chars).items() if freq > args.min_freq]\n",
    "        if is_target:\n",
    "            symbols = ['<pad>','<start>','<end>','<unk>']\n",
    "            return {char: idx for idx, char in enumerate(symbols + chars)}\n",
    "        else:\n",
    "            symbols = ['<pad>','<unk>'] if not args.tied_embedding else ['<pad>','<start>','<end>','<unk>']\n",
    "            return {char: idx for idx, char in enumerate(symbols + chars)}\n",
    "\n",
    "\n",
    "    def pad(self, data, word2idx, max_len, is_target=False):\n",
    "        res = []\n",
    "        for line in data.split('\\n'):\n",
    "            temp_line = [word2idx.get(char, word2idx['<unk>']) for char in line]\n",
    "            if len(temp_line) >= max_len:\n",
    "                if is_target:\n",
    "                    temp_line = temp_line[:(max_len-1)] + [word2idx['<end>']]\n",
    "                else:\n",
    "                    temp_line = temp_line[:max_len]\n",
    "            if len(temp_line) < max_len:\n",
    "                if is_target:\n",
    "                    temp_line += ([word2idx['<end>']] + [word2idx['<pad>']]*(max_len-len(temp_line)-1)) \n",
    "                else:\n",
    "                    temp_line += [word2idx['<pad>']] * (max_len - len(temp_line))\n",
    "            res.append(temp_line)\n",
    "        return np.array(res)\n",
    "\n",
    "\n",
    "    def load(self):\n",
    "        source_idx = self.pad(self.source_words, self.source_word2idx, args.source_max_len)\n",
    "        target_idx = self.pad(self.target_words, self.target_word2idx, args.target_max_len, is_target=True)\n",
    "        return source_idx, target_idx\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def layer_norm(inputs, epsilon=1e-8):\n",
    "    mean, variance = tf.nn.moments(inputs, [-1], keep_dims=True)\n",
    "    normalized = (inputs - mean) / (tf.sqrt(variance + epsilon))\n",
    "\n",
    "    params_shape = inputs.get_shape()[-1:]\n",
    "    gamma = tf.get_variable('gamma', params_shape, tf.float32, tf.ones_initializer())\n",
    "    beta = tf.get_variable('beta', params_shape, tf.float32, tf.zeros_initializer())\n",
    "    \n",
    "    outputs = gamma * normalized + beta\n",
    "    return outputs\n",
    "\n",
    "\n",
    "def embed_seq(inputs, vocab_size, embed_dim, zero_pad=False, scale=False):\n",
    "    lookup_table = tf.get_variable('lookup_table', dtype=tf.float32, shape=[vocab_size, embed_dim])\n",
    "    if zero_pad:\n",
    "        lookup_table = tf.concat((tf.zeros([1, embed_dim]), lookup_table[1:, :]), axis=0)\n",
    "    outputs = tf.nn.embedding_lookup(lookup_table, inputs)\n",
    "    if scale:\n",
    "        outputs = outputs * np.sqrt(embed_dim)\n",
    "    return outputs\n",
    "\n",
    "\n",
    "def multihead_attn(queries, keys, q_masks, k_masks, future_binding, is_training):\n",
    "    \"\"\"\n",
    "    Args:\n",
    "      queries: A 3d tensor with shape of [N, T_q, C_q]\n",
    "      keys: A 3d tensor with shape of [N, T_k, C_k]\n",
    "    \"\"\"\n",
    "    num_units = args.hidden_units\n",
    "    num_heads = args.num_heads\n",
    "    dropout_rate = args.dropout_rate\n",
    "    \n",
    "    T_q = queries.get_shape().as_list()[1]                                         # max time length of query\n",
    "    T_k = keys.get_shape().as_list()[1]                                            # max time length of key\n",
    "\n",
    "    Q = tf.layers.dense(queries, num_units, name='Q')                              # (N, T_q, C)\n",
    "    K_V = tf.layers.dense(keys, 2*num_units, name='K_V')    \n",
    "    K, V = tf.split(K_V, 2, -1)        \n",
    "\n",
    "    Q_ = tf.concat(tf.split(Q, num_heads, axis=2), axis=0)                         # (h*N, T_q, C/h) \n",
    "    K_ = tf.concat(tf.split(K, num_heads, axis=2), axis=0)                         # (h*N, T_k, C/h) \n",
    "    V_ = tf.concat(tf.split(V, num_heads, axis=2), axis=0)                         # (h*N, T_k, C/h)\n",
    "\n",
    "    # Scaled Dot-Product\n",
    "    align = tf.matmul(Q_, tf.transpose(K_, [0,2,1]))                               # (h*N, T_q, T_k)\n",
    "    align = align / np.sqrt(K_.get_shape().as_list()[-1])                          # scale\n",
    "\n",
    "    # Key Masking\n",
    "    paddings = tf.fill(tf.shape(align), float('-inf'))                             # exp(-large) -> 0\n",
    "\n",
    "    key_masks = k_masks                                                            # (N, T_k)\n",
    "    key_masks = tf.tile(key_masks, [num_heads, 1])                                 # (h*N, T_k)\n",
    "    key_masks = tf.tile(tf.expand_dims(key_masks, 1), [1, T_q, 1])                 # (h*N, T_q, T_k)\n",
    "    align = tf.where(tf.equal(key_masks, 0), paddings, align)                      # (h*N, T_q, T_k)\n",
    "\n",
    "    if future_binding:\n",
    "        lower_tri = tf.ones([T_q, T_k])                                            # (T_q, T_k)\n",
    "        lower_tri = tf.linalg.LinearOperatorLowerTriangular(lower_tri).to_dense()  # (T_q, T_k)\n",
    "        masks = tf.tile(tf.expand_dims(lower_tri,0), [tf.shape(align)[0], 1, 1])   # (h*N, T_q, T_k)\n",
    "        align = tf.where(tf.equal(masks, 0), paddings, align)                      # (h*N, T_q, T_k)\n",
    "    \n",
    "    # Softmax\n",
    "    align = tf.nn.softmax(align)                                                   # (h*N, T_q, T_k)\n",
    "\n",
    "    # Query Masking\n",
    "    query_masks = tf.to_float(q_masks)                                             # (N, T_q)\n",
    "    query_masks = tf.tile(query_masks, [num_heads, 1])                             # (h*N, T_q)\n",
    "    query_masks = tf.tile(tf.expand_dims(query_masks, -1), [1, 1, T_k])            # (h*N, T_q, T_k)\n",
    "    align *= query_masks                                                           # (h*N, T_q, T_k)\n",
    "\n",
    "    align = tf.layers.dropout(align, dropout_rate, training=is_training)           # (h*N, T_q, T_k)\n",
    "\n",
    "    # Weighted sum\n",
    "    outputs = tf.matmul(align, V_)                                                 # (h*N, T_q, C/h)\n",
    "    # Restore shape\n",
    "    outputs = tf.concat(tf.split(outputs, num_heads, axis=0), axis=2)              # (N, T_q, C)\n",
    "    # Residual connection\n",
    "    outputs += queries                                                             # (N, T_q, C)   \n",
    "    # Normalize\n",
    "    outputs = layer_norm(outputs)                                                  # (N, T_q, C)\n",
    "    return outputs\n",
    "\n",
    "\n",
    "def pointwise_feedforward(inputs, activation=None):\n",
    "    num_units = [4*args.hidden_units, args.hidden_units]\n",
    "    # Inner layer\n",
    "    outputs = tf.layers.conv1d(inputs, num_units[0], kernel_size=1, activation=activation)\n",
    "    # Readout layer\n",
    "    outputs = tf.layers.conv1d(outputs, num_units[1], kernel_size=1, activation=None)\n",
    "    # Residual connection\n",
    "    outputs += inputs\n",
    "    # Normalize\n",
    "    outputs = layer_norm(outputs)\n",
    "    return outputs\n",
    "\n",
    "\n",
    "def learned_position_encoding(inputs, mask, embed_dim):\n",
    "    T = inputs.get_shape().as_list()[1]\n",
    "    outputs = tf.range(tf.shape(inputs)[1])                # (T_q)\n",
    "    outputs = tf.expand_dims(outputs, 0)                   # (1, T_q)\n",
    "    outputs = tf.tile(outputs, [tf.shape(inputs)[0], 1])   # (N, T_q)\n",
    "    outputs = embed_seq(outputs, T, embed_dim, zero_pad=False, scale=False)\n",
    "    return tf.expand_dims(tf.to_float(mask), -1) * outputs\n",
    "\n",
    "\n",
    "def sinusoidal_position_encoding(inputs, mask, repr_dim):\n",
    "    T = inputs.get_shape()[1].value\n",
    "    pos = tf.reshape(tf.range(0.0, tf.to_float(T), dtype=tf.float32), [-1, 1])\n",
    "    i = np.arange(0, repr_dim, 2, np.float32)\n",
    "    denom = np.reshape(np.power(10000.0, i / repr_dim), [1, -1])\n",
    "    enc = tf.expand_dims(tf.concat([tf.sin(pos / denom), tf.cos(pos / denom)], 1), 0)\n",
    "    return tf.tile(enc, [tf.shape(inputs)[0], 1, 1]) * tf.expand_dims(tf.to_float(mask), -1)\n",
    "\n",
    "\n",
    "def label_smoothing(inputs, epsilon=0.1):\n",
    "    C = inputs.get_shape().as_list()[-1] # number of channels\n",
    "    return ((1 - epsilon) * inputs) + (epsilon / C)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward_pass(sources, targets, mode, params):\n",
    "    is_training = (mode == tf.estimator.ModeKeys.TRAIN)\n",
    "    pos_enc = _get_position_encoder()\n",
    "\n",
    "    # ENCODER\n",
    "    en_masks = tf.sign(sources)   \n",
    "\n",
    "    with tf.variable_scope('encoder_embedding'):\n",
    "        encoded = embed_seq(inputs = sources,\n",
    "                            vocab_size = params['source_vocab_size'],\n",
    "                            embed_dim = args.hidden_units,\n",
    "                            zero_pad = True,\n",
    "                            scale = True)\n",
    "\n",
    "    with tf.variable_scope('encoder_position_encoding'):\n",
    "        encoded += pos_enc(sources, en_masks, args.hidden_units)\n",
    "\n",
    "    with tf.variable_scope('encoder_dropout'):\n",
    "        encoded = tf.layers.dropout(encoded, args.dropout_rate, training=is_training)\n",
    "\n",
    "    for i in range(args.num_blocks):\n",
    "        with tf.variable_scope('encoder_attn_%d'%i):\n",
    "            encoded = multihead_attn(queries = encoded,\n",
    "                                     keys = encoded,\n",
    "                                     q_masks = en_masks,\n",
    "                                     k_masks = en_masks,\n",
    "                                     future_binding = False,\n",
    "                                     is_training = is_training,)\n",
    "\n",
    "        with tf.variable_scope('encoder_feedforward_%d'%i):\n",
    "            encoded = pointwise_feedforward(encoded,\n",
    "                                            activation = params['activation'])\n",
    "\n",
    "    # DECODER\n",
    "    decoder_inputs = _shift_right(targets, params['start_symbol'])\n",
    "    de_masks = tf.sign(decoder_inputs)\n",
    "\n",
    "    if args.tied_embedding:\n",
    "        vs = tf.variable_scope('encoder_embedding', reuse=True)\n",
    "    else:\n",
    "        vs = tf.variable_scope('decoder_embedding')\n",
    "    with vs:\n",
    "        decoded = embed_seq(decoder_inputs,\n",
    "                            params['target_vocab_size'],\n",
    "                            args.hidden_units,\n",
    "                            zero_pad = True,\n",
    "                            scale = True)\n",
    "\n",
    "    with tf.variable_scope('decoder_position_encoding'):\n",
    "        decoded += pos_enc(decoder_inputs, de_masks, args.hidden_units)\n",
    "\n",
    "    with tf.variable_scope('decoder_dropout'):\n",
    "        decoded = tf.layers.dropout(decoded, args.dropout_rate, training=is_training)\n",
    "\n",
    "    for i in range(args.num_blocks):\n",
    "        with tf.variable_scope('decoder_self_attn_%d'%i):\n",
    "            decoded = multihead_attn(queries = decoded,\n",
    "                                     keys = decoded,\n",
    "                                     q_masks = de_masks,\n",
    "                                     k_masks = de_masks,\n",
    "                                     future_binding = True,\n",
    "                                     is_training = is_training)\n",
    "\n",
    "        with tf.variable_scope('decoder_attn_%d'%i):\n",
    "            decoded = multihead_attn(queries=decoded,\n",
    "                                     keys = encoded,\n",
    "                                     q_masks = de_masks,\n",
    "                                     k_masks = en_masks,\n",
    "                                     future_binding = False,\n",
    "                                     is_training = is_training)\n",
    "\n",
    "        with tf.variable_scope('decoder_feedforward_%d'%i):\n",
    "            decoded = pointwise_feedforward(decoded,\n",
    "                                            activation = params['activation'])\n",
    "\n",
    "    # OUTPUT LAYER    \n",
    "    if args.tied_proj_weight:\n",
    "        b = tf.get_variable('bias', [params['target_vocab_size']], tf.float32)\n",
    "        _scope = 'encoder_embedding' if args.tied_embedding else 'decoder_embedding'\n",
    "        with tf.variable_scope(_scope, reuse=True):\n",
    "            shared_w = tf.get_variable('lookup_table')\n",
    "        decoded = tf.reshape(decoded, [-1, args.hidden_units])\n",
    "        logits = tf.nn.xw_plus_b(decoded, tf.transpose(shared_w), b)\n",
    "        logits = tf.reshape(logits, [tf.shape(sources)[0], -1, params['target_vocab_size']])\n",
    "    else:\n",
    "        with tf.variable_scope('output_layer'):\n",
    "            logits = tf.layers.dense(decoded, params['target_vocab_size'], reuse=reuse)\n",
    "    return logits\n",
    "\n",
    "\n",
    "def _model_fn_train(features, mode, params):\n",
    "    logits = forward_pass(features['source'], features['target'], mode, params)\n",
    "    targets = features['target']\n",
    "    masks = tf.to_float(tf.not_equal(targets, 0))\n",
    "\n",
    "    if args.label_smoothing:\n",
    "        loss_op = label_smoothing_sequence_loss(logits = logits,\n",
    "                                                targets = targets,\n",
    "                                                weights = masks,\n",
    "                                                label_depth = params['target_vocab_size'])\n",
    "    else:\n",
    "        loss_op = tf.contrib.seq2seq.sequence_loss(logits = logits,\n",
    "                                                   targets = targets,\n",
    "                                                   weights = masks)\n",
    "\n",
    "    if args.lr_decay_strategy == 'noam':\n",
    "        step_num = tf.train.get_global_step() + 1   # prevents zero global step\n",
    "        lr = _get_noam_lr(step_num)\n",
    "    elif args.lr_decay_strategy == 'exp':\n",
    "        lr = tf.train.exponential_decay(1e-3, tf.train.get_global_step(), 100000, 0.1)\n",
    "    else:\n",
    "        raise ValueError(\"lr decay strategy must be one of 'noam' and 'exp'\")\n",
    "    log_hook = tf.train.LoggingTensorHook({'lr': lr}, every_n_iter=100)\n",
    "\n",
    "    train_op = tf.train.AdamOptimizer(lr).minimize(loss_op,\n",
    "                                                   global_step = tf.train.get_global_step())\n",
    "    \n",
    "    return tf.estimator.EstimatorSpec(mode = mode,\n",
    "                                      loss = loss_op,\n",
    "                                      train_op = train_op,\n",
    "                                      training_hooks = [log_hook])\n",
    "\n",
    "\n",
    "def _model_fn_predict(features, mode, params):\n",
    "    def cond(i, x, temp):\n",
    "        return i < args.target_max_len\n",
    "\n",
    "    def body(i, x, temp):\n",
    "        logits = forward_pass(features['source'], x, mode, params)\n",
    "        ids = tf.argmax(logits, -1)[:, i]\n",
    "        ids = tf.expand_dims(ids, -1)\n",
    "\n",
    "        temp = tf.concat([temp[:, 1:], ids], -1)\n",
    "\n",
    "        x = tf.concat([temp[:, -(i+1):], temp[:, :-(i+1)]], -1)\n",
    "        x = tf.reshape(x, [tf.shape(temp)[0], args.target_max_len])\n",
    "        i += 1\n",
    "        return i, x, temp\n",
    "\n",
    "    _, res, _ = tf.while_loop(cond, body, [tf.constant(0), features['target'], features['target']])\n",
    "    \n",
    "    return tf.estimator.EstimatorSpec(mode=mode, predictions=res)\n",
    "\n",
    "\n",
    "def tf_estimator_model_fn(features, labels, mode, params):\n",
    "    if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "        return _model_fn_train(features, mode, params)\n",
    "    if mode == tf.estimator.ModeKeys.PREDICT:\n",
    "        return _model_fn_predict(features, mode, params)\n",
    "\n",
    "\n",
    "def _shift_right(targets, start_symbol):\n",
    "    start_symbols = tf.cast(tf.fill([tf.shape(targets)[0], 1], start_symbol), tf.int64)\n",
    "    return tf.concat([start_symbols, targets[:, :-1]], axis=-1)\n",
    "\n",
    "\n",
    "def _get_position_encoder():\n",
    "    if args.position_encoding == 'non_param':\n",
    "        pos_enc = sinusoidal_position_encoding\n",
    "    elif args.position_encoding == 'param':\n",
    "        pos_enc = learned_position_encoding\n",
    "    else:\n",
    "        raise ValueError(\"position encoding has to be either 'param' or 'non_param'\")\n",
    "    return pos_enc\n",
    "\n",
    "\n",
    "def _get_noam_lr(step_num):\n",
    "    return tf.rsqrt(tf.to_float(args.hidden_units)) * tf.minimum(\n",
    "        tf.rsqrt(tf.to_float(step_num)),\n",
    "        tf.to_float(step_num) * tf.convert_to_tensor(args.warmup_steps ** (-1.5)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Source Vocab Size: 2022\n",
      "Target Vocab Size: 2481\n",
      "INFO:tensorflow:Using default config.\n",
      "WARNING:tensorflow:Using temporary folder as model directory: /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpe6f5vo_q\n",
      "INFO:tensorflow:Using config: {'_model_dir': '/var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpe6f5vo_q', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x11f77e5f8>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpe6f5vo_q/model.ckpt.\n",
      "INFO:tensorflow:loss = 7.74742, step = 1\n",
      "INFO:tensorflow:lr = 0.001\n",
      "INFO:tensorflow:global_step/sec: 5.78924\n",
      "INFO:tensorflow:loss = 4.952749, step = 101 (17.275 sec)\n",
      "INFO:tensorflow:lr = 0.0009977 (17.274 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.05594\n",
      "INFO:tensorflow:loss = 4.2536483, step = 201 (16.513 sec)\n",
      "INFO:tensorflow:lr = 0.0009954055 (16.513 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.53248\n",
      "INFO:tensorflow:loss = 4.5511775, step = 301 (18.075 sec)\n",
      "INFO:tensorflow:lr = 0.0009931161 (18.075 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.88602\n",
      "INFO:tensorflow:loss = 4.020638, step = 401 (16.990 sec)\n",
      "INFO:tensorflow:lr = 0.000990832 (16.990 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.72575\n",
      "INFO:tensorflow:loss = 4.0278635, step = 501 (17.465 sec)\n",
      "INFO:tensorflow:lr = 0.0009885532 (17.465 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.98481\n",
      "INFO:tensorflow:loss = 3.7466712, step = 601 (16.709 sec)\n",
      "INFO:tensorflow:lr = 0.0009862796 (16.709 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.8787\n",
      "INFO:tensorflow:loss = 3.8569446, step = 701 (17.010 sec)\n",
      "INFO:tensorflow:lr = 0.0009840112 (17.010 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.51924\n",
      "INFO:tensorflow:loss = 4.080703, step = 801 (18.119 sec)\n",
      "INFO:tensorflow:lr = 0.000981748 (18.119 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.64834\n",
      "INFO:tensorflow:loss = 3.6957686, step = 901 (17.704 sec)\n",
      "INFO:tensorflow:lr = 0.00097949 (17.704 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.84117\n",
      "INFO:tensorflow:loss = 3.7106626, step = 1001 (17.120 sec)\n",
      "INFO:tensorflow:lr = 0.0009772372 (17.120 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.8715\n",
      "INFO:tensorflow:loss = 3.7429078, step = 1101 (17.031 sec)\n",
      "INFO:tensorflow:lr = 0.00097498967 (17.031 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.6776\n",
      "INFO:tensorflow:loss = 3.5386598, step = 1201 (17.614 sec)\n",
      "INFO:tensorflow:lr = 0.0009727473 (17.614 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.89005\n",
      "INFO:tensorflow:loss = 3.7325912, step = 1301 (16.977 sec)\n",
      "INFO:tensorflow:lr = 0.00097051 (16.976 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.9609\n",
      "INFO:tensorflow:loss = 3.7814069, step = 1401 (16.776 sec)\n",
      "INFO:tensorflow:lr = 0.0009682779 (16.776 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.95678\n",
      "INFO:tensorflow:loss = 3.5961285, step = 1501 (16.788 sec)\n",
      "INFO:tensorflow:lr = 0.0009660509 (16.788 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.58312\n",
      "INFO:tensorflow:loss = 3.4271338, step = 1601 (17.912 sec)\n",
      "INFO:tensorflow:lr = 0.0009638291 (17.912 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.27622\n",
      "INFO:tensorflow:loss = 3.4458807, step = 1701 (18.953 sec)\n",
      "INFO:tensorflow:lr = 0.0009616123 (18.952 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.72518\n",
      "INFO:tensorflow:loss = 3.2768762, step = 1801 (17.467 sec)\n",
      "INFO:tensorflow:lr = 0.0009594007 (17.467 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.79024\n",
      "INFO:tensorflow:loss = 3.2271435, step = 1901 (17.270 sec)\n",
      "INFO:tensorflow:lr = 0.00095719413 (17.270 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.8\n",
      "INFO:tensorflow:loss = 3.3596687, step = 2001 (17.241 sec)\n",
      "INFO:tensorflow:lr = 0.00095499266 (17.242 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.53476\n",
      "INFO:tensorflow:loss = 3.1568215, step = 2101 (18.068 sec)\n",
      "INFO:tensorflow:lr = 0.0009527962 (18.068 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.80285\n",
      "INFO:tensorflow:loss = 3.3921645, step = 2201 (17.233 sec)\n",
      "INFO:tensorflow:lr = 0.00095060485 (17.233 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.87624\n",
      "INFO:tensorflow:loss = 3.5421703, step = 2301 (17.018 sec)\n",
      "INFO:tensorflow:lr = 0.00094841846 (17.018 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.11861\n",
      "INFO:tensorflow:loss = 3.5788276, step = 2401 (16.344 sec)\n",
      "INFO:tensorflow:lr = 0.0009462372 (16.344 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.92595\n",
      "INFO:tensorflow:loss = 3.4993646, step = 2501 (16.875 sec)\n",
      "INFO:tensorflow:lr = 0.0009440609 (16.875 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.00064\n",
      "INFO:tensorflow:loss = 3.2875254, step = 2601 (16.665 sec)\n",
      "INFO:tensorflow:lr = 0.00094188965 (16.665 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.08491\n",
      "INFO:tensorflow:loss = 3.0971951, step = 2701 (16.434 sec)\n",
      "INFO:tensorflow:lr = 0.00093972334 (16.434 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.59519\n",
      "INFO:tensorflow:loss = 2.4909503, step = 2801 (17.873 sec)\n",
      "INFO:tensorflow:lr = 0.000937562 (17.873 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.42323\n",
      "INFO:tensorflow:loss = 3.1751263, step = 2901 (18.439 sec)\n",
      "INFO:tensorflow:lr = 0.0009354057 (18.439 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.91123\n",
      "INFO:tensorflow:loss = 2.9099505, step = 3001 (16.917 sec)\n",
      "INFO:tensorflow:lr = 0.00093325437 (16.917 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.68159\n",
      "INFO:tensorflow:loss = 3.0477045, step = 3101 (17.600 sec)\n",
      "INFO:tensorflow:lr = 0.0009311079 (17.600 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.45966\n",
      "INFO:tensorflow:loss = 2.9709556, step = 3201 (18.316 sec)\n",
      "INFO:tensorflow:lr = 0.00092896644 (18.316 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.81231\n",
      "INFO:tensorflow:loss = 3.3570054, step = 3301 (17.205 sec)\n",
      "INFO:tensorflow:lr = 0.0009268299 (17.205 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.40873\n",
      "INFO:tensorflow:loss = 2.457609, step = 3401 (18.489 sec)\n",
      "INFO:tensorflow:lr = 0.0009246982 (18.489 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 3430 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpe6f5vo_q/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 4.24549\n",
      "INFO:tensorflow:loss = 3.3839242, step = 3501 (23.555 sec)\n",
      "INFO:tensorflow:lr = 0.00092257146 (23.555 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.75426\n",
      "INFO:tensorflow:loss = 3.6485894, step = 3601 (17.378 sec)\n",
      "INFO:tensorflow:lr = 0.0009204496 (17.378 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.4927\n",
      "INFO:tensorflow:loss = 3.5267642, step = 3701 (18.206 sec)\n",
      "INFO:tensorflow:lr = 0.0009183326 (18.207 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.43686\n",
      "INFO:tensorflow:loss = 3.0439513, step = 3801 (18.393 sec)\n",
      "INFO:tensorflow:lr = 0.00091622054 (18.393 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.46603\n",
      "INFO:tensorflow:loss = 2.831337, step = 3901 (18.294 sec)\n",
      "INFO:tensorflow:lr = 0.00091411325 (18.293 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.58336\n",
      "INFO:tensorflow:loss = 2.8198123, step = 4001 (17.911 sec)\n",
      "INFO:tensorflow:lr = 0.0009120109 (17.910 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.63714\n",
      "INFO:tensorflow:loss = 2.5915835, step = 4101 (17.739 sec)\n",
      "INFO:tensorflow:lr = 0.00090991333 (17.740 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.52698\n",
      "INFO:tensorflow:loss = 2.5759895, step = 4201 (18.093 sec)\n",
      "INFO:tensorflow:lr = 0.0009078206 (18.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.03924\n",
      "INFO:tensorflow:loss = 3.0705905, step = 4301 (16.558 sec)\n",
      "INFO:tensorflow:lr = 0.0009057327 (16.558 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.6874\n",
      "INFO:tensorflow:loss = 2.7792902, step = 4401 (17.583 sec)\n",
      "INFO:tensorflow:lr = 0.0009036495 (17.583 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.25128\n",
      "INFO:tensorflow:loss = 2.984917, step = 4501 (19.043 sec)\n",
      "INFO:tensorflow:lr = 0.0009015712 (19.043 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.68775\n",
      "INFO:tensorflow:loss = 3.4734058, step = 4601 (17.581 sec)\n",
      "INFO:tensorflow:lr = 0.0008994976 (17.581 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.59504\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = 3.3355758, step = 4701 (17.873 sec)\n",
      "INFO:tensorflow:lr = 0.00089742884 (17.873 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.21484\n",
      "INFO:tensorflow:loss = 3.3406794, step = 4801 (16.090 sec)\n",
      "INFO:tensorflow:lr = 0.0008953648 (16.090 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.37646\n",
      "INFO:tensorflow:loss = 3.1371355, step = 4901 (18.600 sec)\n",
      "INFO:tensorflow:lr = 0.0008933055 (18.600 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.86336\n",
      "INFO:tensorflow:loss = 3.4194505, step = 5001 (17.055 sec)\n",
      "INFO:tensorflow:lr = 0.000891251 (17.054 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.98557\n",
      "INFO:tensorflow:loss = 3.142365, step = 5101 (16.707 sec)\n",
      "INFO:tensorflow:lr = 0.0008892011 (16.707 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.87209\n",
      "INFO:tensorflow:loss = 3.5709221, step = 5201 (17.030 sec)\n",
      "INFO:tensorflow:lr = 0.00088715606 (17.030 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.69939\n",
      "INFO:tensorflow:loss = 3.7552063, step = 5301 (17.546 sec)\n",
      "INFO:tensorflow:lr = 0.00088511565 (17.546 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.88801\n",
      "INFO:tensorflow:loss = 3.5339081, step = 5401 (16.983 sec)\n",
      "INFO:tensorflow:lr = 0.00088307995 (16.983 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.13169\n",
      "INFO:tensorflow:loss = 3.5862486, step = 5501 (16.310 sec)\n",
      "INFO:tensorflow:lr = 0.0008810489 (16.310 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.32773\n",
      "INFO:tensorflow:loss = 3.1284604, step = 5601 (15.802 sec)\n",
      "INFO:tensorflow:lr = 0.0008790226 (15.803 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.34516\n",
      "INFO:tensorflow:loss = 3.176029, step = 5701 (15.760 sec)\n",
      "INFO:tensorflow:lr = 0.00087700086 (15.760 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.69582\n",
      "INFO:tensorflow:loss = 3.1532433, step = 5801 (17.557 sec)\n",
      "INFO:tensorflow:lr = 0.00087498385 (17.557 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.58166\n",
      "INFO:tensorflow:loss = 3.3363044, step = 5901 (17.915 sec)\n",
      "INFO:tensorflow:lr = 0.0008729714 (17.915 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.79523\n",
      "INFO:tensorflow:loss = 3.4740944, step = 6001 (17.256 sec)\n",
      "INFO:tensorflow:lr = 0.0008709636 (17.257 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.09119\n",
      "INFO:tensorflow:loss = 3.6398098, step = 6101 (16.417 sec)\n",
      "INFO:tensorflow:lr = 0.00086896046 (16.416 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.13755\n",
      "INFO:tensorflow:loss = 3.1516702, step = 6201 (16.293 sec)\n",
      "INFO:tensorflow:lr = 0.00086696196 (16.293 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.10933\n",
      "INFO:tensorflow:loss = 3.3897045, step = 6301 (16.368 sec)\n",
      "INFO:tensorflow:lr = 0.000864968 (16.368 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.16519\n",
      "INFO:tensorflow:loss = 3.5506618, step = 6401 (16.220 sec)\n",
      "INFO:tensorflow:lr = 0.00086297863 (16.220 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.84058\n",
      "INFO:tensorflow:loss = 3.1143138, step = 6501 (17.122 sec)\n",
      "INFO:tensorflow:lr = 0.0008609938 (17.122 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.86419\n",
      "INFO:tensorflow:loss = 3.3201137, step = 6601 (17.053 sec)\n",
      "INFO:tensorflow:lr = 0.0008590135 (17.053 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.0046\n",
      "INFO:tensorflow:loss = 3.6605892, step = 6701 (16.654 sec)\n",
      "INFO:tensorflow:lr = 0.0008570379 (16.653 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.05514\n",
      "INFO:tensorflow:loss = 3.4081628, step = 6801 (16.515 sec)\n",
      "INFO:tensorflow:lr = 0.00085506676 (16.515 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 6875 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpe6f5vo_q/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 4.3221\n",
      "INFO:tensorflow:loss = 3.369681, step = 6901 (23.137 sec)\n",
      "INFO:tensorflow:lr = 0.00085310015 (23.137 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.5009\n",
      "INFO:tensorflow:loss = 2.9709895, step = 7001 (15.382 sec)\n",
      "INFO:tensorflow:lr = 0.0008511381 (15.382 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 7092 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpe6f5vo_q/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 3.1559217.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpe6f5vo_q/model.ckpt-7092\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "你是谁 -> 我是小通\n",
      "你喜欢我吗 -> 喜欢你\n",
      "给我唱一首歌 -> =。==。=。=\n",
      "我帅吗 -> =。==。=。=\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpe6f5vo_q/model.ckpt-7092\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 7093 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpe6f5vo_q/model.ckpt.\n",
      "INFO:tensorflow:loss = 3.0702608, step = 7093\n",
      "INFO:tensorflow:lr = 0.00084933697\n",
      "INFO:tensorflow:global_step/sec: 5.59412\n",
      "INFO:tensorflow:loss = 2.9289117, step = 7193 (17.877 sec)\n",
      "INFO:tensorflow:lr = 0.0008473835 (17.877 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.47986\n",
      "INFO:tensorflow:loss = 2.4817665, step = 7293 (15.432 sec)\n",
      "INFO:tensorflow:lr = 0.00084543467 (15.432 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.31718\n",
      "INFO:tensorflow:loss = 3.0721374, step = 7393 (15.830 sec)\n",
      "INFO:tensorflow:lr = 0.0008434902 (15.830 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.02673\n",
      "INFO:tensorflow:loss = 2.614206, step = 7493 (16.594 sec)\n",
      "INFO:tensorflow:lr = 0.00084155024 (16.594 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.94246\n",
      "INFO:tensorflow:loss = 2.6714737, step = 7593 (16.827 sec)\n",
      "INFO:tensorflow:lr = 0.00083961466 (16.827 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.89477\n",
      "INFO:tensorflow:loss = 2.6814702, step = 7693 (16.964 sec)\n",
      "INFO:tensorflow:lr = 0.0008376836 (16.964 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.85964\n",
      "INFO:tensorflow:loss = 2.6138103, step = 7793 (17.066 sec)\n",
      "INFO:tensorflow:lr = 0.000835757 (17.066 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.98013\n",
      "INFO:tensorflow:loss = 2.8796518, step = 7893 (16.722 sec)\n",
      "INFO:tensorflow:lr = 0.0008338348 (16.722 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.2644\n",
      "INFO:tensorflow:loss = 3.5398397, step = 7993 (15.963 sec)\n",
      "INFO:tensorflow:lr = 0.00083191704 (15.963 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.14266\n",
      "INFO:tensorflow:loss = 2.9089997, step = 8093 (16.280 sec)\n",
      "INFO:tensorflow:lr = 0.0008300037 (16.280 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.50552\n",
      "INFO:tensorflow:loss = 2.872046, step = 8193 (15.372 sec)\n",
      "INFO:tensorflow:lr = 0.0008280948 (15.371 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.47518\n",
      "INFO:tensorflow:loss = 2.8939843, step = 8293 (15.444 sec)\n",
      "INFO:tensorflow:lr = 0.00082619017 (15.444 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.52028\n",
      "INFO:tensorflow:loss = 2.3900266, step = 8393 (15.337 sec)\n",
      "INFO:tensorflow:lr = 0.00082429004 (15.337 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.49917\n",
      "INFO:tensorflow:loss = 2.9035764, step = 8493 (15.387 sec)\n",
      "INFO:tensorflow:lr = 0.00082239415 (15.387 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.41432\n",
      "INFO:tensorflow:loss = 2.3562562, step = 8593 (15.590 sec)\n",
      "INFO:tensorflow:lr = 0.00082050276 (15.591 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.43931\n",
      "INFO:tensorflow:loss = 2.7565274, step = 8693 (15.529 sec)\n",
      "INFO:tensorflow:lr = 0.0008186156 (15.529 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.12621\n",
      "INFO:tensorflow:loss = 2.7602668, step = 8793 (16.324 sec)\n",
      "INFO:tensorflow:lr = 0.0008167329 (16.323 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.71181\n",
      "INFO:tensorflow:loss = 2.798641, step = 8893 (17.507 sec)\n",
      "INFO:tensorflow:lr = 0.0008148544 (17.507 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.78405\n",
      "INFO:tensorflow:loss = 2.7018309, step = 8993 (17.289 sec)\n",
      "INFO:tensorflow:lr = 0.0008129803 (17.289 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.62962\n",
      "INFO:tensorflow:loss = 3.242074, step = 9093 (17.763 sec)\n",
      "INFO:tensorflow:lr = 0.0008111105 (17.763 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.73122\n",
      "INFO:tensorflow:loss = 3.087495, step = 9193 (17.448 sec)\n",
      "INFO:tensorflow:lr = 0.00080924504 (17.448 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.56458\n",
      "INFO:tensorflow:loss = 2.958401, step = 9293 (17.971 sec)\n",
      "INFO:tensorflow:lr = 0.0008073838 (17.972 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.55297\n",
      "INFO:tensorflow:loss = 3.1255767, step = 9393 (18.008 sec)\n",
      "INFO:tensorflow:lr = 0.00080552686 (18.008 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.94302\n",
      "INFO:tensorflow:loss = 3.172246, step = 9493 (16.826 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:lr = 0.0008036742 (16.826 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.88716\n",
      "INFO:tensorflow:loss = 3.2470222, step = 9593 (16.986 sec)\n",
      "INFO:tensorflow:lr = 0.0008018258 (16.986 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.94453\n",
      "INFO:tensorflow:loss = 3.494174, step = 9693 (16.822 sec)\n",
      "INFO:tensorflow:lr = 0.00079998164 (16.823 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.12317\n",
      "INFO:tensorflow:loss = 3.5742495, step = 9793 (16.331 sec)\n",
      "INFO:tensorflow:lr = 0.00079814176 (16.331 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.01196\n",
      "INFO:tensorflow:loss = 3.3162043, step = 9893 (16.634 sec)\n",
      "INFO:tensorflow:lr = 0.00079630606 (16.634 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.90076\n",
      "INFO:tensorflow:loss = 3.1549158, step = 9993 (16.947 sec)\n",
      "INFO:tensorflow:lr = 0.0007944746 (16.947 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.32924\n",
      "INFO:tensorflow:loss = 3.243188, step = 10093 (15.800 sec)\n",
      "INFO:tensorflow:lr = 0.00079264736 (15.799 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.8758\n",
      "INFO:tensorflow:loss = 2.9707773, step = 10193 (17.019 sec)\n",
      "INFO:tensorflow:lr = 0.00079082436 (17.019 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.94278\n",
      "INFO:tensorflow:loss = 3.255025, step = 10293 (16.827 sec)\n",
      "INFO:tensorflow:lr = 0.0007890055 (16.827 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.426\n",
      "INFO:tensorflow:loss = 3.1281044, step = 10393 (15.562 sec)\n",
      "INFO:tensorflow:lr = 0.00078719086 (15.562 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.52013\n",
      "INFO:tensorflow:loss = 3.2770226, step = 10493 (18.116 sec)\n",
      "INFO:tensorflow:lr = 0.00078538037 (18.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.78918\n",
      "INFO:tensorflow:loss = 2.8377392, step = 10593 (17.273 sec)\n",
      "INFO:tensorflow:lr = 0.000783574 (17.273 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 10677 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpe6f5vo_q/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 4.63482\n",
      "INFO:tensorflow:loss = 3.379755, step = 10693 (21.576 sec)\n",
      "INFO:tensorflow:lr = 0.00078177184 (21.576 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.21868\n",
      "INFO:tensorflow:loss = 3.2435238, step = 10793 (16.081 sec)\n",
      "INFO:tensorflow:lr = 0.00077997387 (16.081 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.9759\n",
      "INFO:tensorflow:loss = 3.218009, step = 10893 (16.734 sec)\n",
      "INFO:tensorflow:lr = 0.0007781799 (16.734 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.7873\n",
      "INFO:tensorflow:loss = 3.2674184, step = 10993 (17.279 sec)\n",
      "INFO:tensorflow:lr = 0.0007763902 (17.279 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.27271\n",
      "INFO:tensorflow:loss = 2.886909, step = 11093 (15.942 sec)\n",
      "INFO:tensorflow:lr = 0.00077460456 (15.942 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.22328\n",
      "INFO:tensorflow:loss = 2.8900936, step = 11193 (16.069 sec)\n",
      "INFO:tensorflow:lr = 0.00077282294 (16.069 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.21061\n",
      "INFO:tensorflow:loss = 2.9409637, step = 11293 (16.101 sec)\n",
      "INFO:tensorflow:lr = 0.00077104557 (16.101 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.32625\n",
      "INFO:tensorflow:loss = 3.1661456, step = 11393 (15.807 sec)\n",
      "INFO:tensorflow:lr = 0.0007692722 (15.807 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.89264\n",
      "INFO:tensorflow:loss = 2.7944722, step = 11493 (16.970 sec)\n",
      "INFO:tensorflow:lr = 0.0007675029 (16.970 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.54916\n",
      "INFO:tensorflow:loss = 3.094459, step = 11593 (18.021 sec)\n",
      "INFO:tensorflow:lr = 0.00076573767 (18.021 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.96689\n",
      "INFO:tensorflow:loss = 3.1050436, step = 11693 (16.759 sec)\n",
      "INFO:tensorflow:lr = 0.00076397654 (16.759 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.95199\n",
      "INFO:tensorflow:loss = 3.2310154, step = 11793 (16.801 sec)\n",
      "INFO:tensorflow:lr = 0.0007622195 (16.801 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.73051\n",
      "INFO:tensorflow:loss = 3.0816207, step = 11893 (17.450 sec)\n",
      "INFO:tensorflow:lr = 0.0007604664 (17.450 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.76247\n",
      "INFO:tensorflow:loss = 3.035883, step = 11993 (17.354 sec)\n",
      "INFO:tensorflow:lr = 0.00075871736 (17.354 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.87408\n",
      "INFO:tensorflow:loss = 3.2625775, step = 12093 (17.024 sec)\n",
      "INFO:tensorflow:lr = 0.00075697235 (17.024 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.73046\n",
      "INFO:tensorflow:loss = 3.0103748, step = 12193 (17.451 sec)\n",
      "INFO:tensorflow:lr = 0.00075523136 (17.451 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.77779\n",
      "INFO:tensorflow:loss = 2.9930916, step = 12293 (17.308 sec)\n",
      "INFO:tensorflow:lr = 0.00075349445 (17.308 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.40679\n",
      "INFO:tensorflow:loss = 2.6405349, step = 12393 (18.495 sec)\n",
      "INFO:tensorflow:lr = 0.0007517614 (18.495 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.78789\n",
      "INFO:tensorflow:loss = 2.8684335, step = 12493 (17.277 sec)\n",
      "INFO:tensorflow:lr = 0.0007500324 (17.277 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.52642\n",
      "INFO:tensorflow:loss = 2.8952324, step = 12593 (18.095 sec)\n",
      "INFO:tensorflow:lr = 0.0007483074 (18.095 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.38298\n",
      "INFO:tensorflow:loss = 3.0109184, step = 12693 (18.577 sec)\n",
      "INFO:tensorflow:lr = 0.0007465863 (18.577 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.13805\n",
      "INFO:tensorflow:loss = 2.9148383, step = 12793 (16.292 sec)\n",
      "INFO:tensorflow:lr = 0.00074486923 (16.292 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.32736\n",
      "INFO:tensorflow:loss = 2.8390913, step = 12893 (15.804 sec)\n",
      "INFO:tensorflow:lr = 0.00074315607 (15.805 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.06466\n",
      "INFO:tensorflow:loss = 3.2881503, step = 12993 (16.490 sec)\n",
      "INFO:tensorflow:lr = 0.0007414469 (16.491 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.17809\n",
      "INFO:tensorflow:loss = 2.875316, step = 13093 (16.185 sec)\n",
      "INFO:tensorflow:lr = 0.0007397416 (16.184 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.38806\n",
      "INFO:tensorflow:loss = 2.9161942, step = 13193 (15.654 sec)\n",
      "INFO:tensorflow:lr = 0.00073804025 (15.654 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.72559\n",
      "INFO:tensorflow:loss = 3.087389, step = 13293 (17.466 sec)\n",
      "INFO:tensorflow:lr = 0.00073634274 (17.466 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.7782\n",
      "INFO:tensorflow:loss = 2.7329118, step = 13393 (17.306 sec)\n",
      "INFO:tensorflow:lr = 0.0007346492 (17.306 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.27859\n",
      "INFO:tensorflow:loss = 2.7157881, step = 13493 (15.927 sec)\n",
      "INFO:tensorflow:lr = 0.00073295954 (15.927 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.98043\n",
      "INFO:tensorflow:loss = 3.078409, step = 13593 (16.721 sec)\n",
      "INFO:tensorflow:lr = 0.0007312738 (16.721 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.71455\n",
      "INFO:tensorflow:loss = 2.679492, step = 13693 (17.499 sec)\n",
      "INFO:tensorflow:lr = 0.00072959193 (17.499 sec)\n",
      "INFO:tensorflow:global_step/sec: 6.0584\n",
      "INFO:tensorflow:loss = 3.017799, step = 13793 (16.506 sec)\n",
      "INFO:tensorflow:lr = 0.0007279139 (16.505 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.96385\n",
      "INFO:tensorflow:loss = 2.7451003, step = 13893 (16.768 sec)\n",
      "INFO:tensorflow:lr = 0.00072623976 (16.769 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.67433\n",
      "INFO:tensorflow:loss = 2.8760397, step = 13993 (17.623 sec)\n",
      "INFO:tensorflow:lr = 0.0007245695 (17.623 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.43354\n",
      "INFO:tensorflow:loss = 2.4893403, step = 14093 (18.404 sec)\n",
      "INFO:tensorflow:lr = 0.000722903 (18.404 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 14180 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpe6f5vo_q/model.ckpt.\n",
      "INFO:tensorflow:Saving checkpoints for 14184 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpe6f5vo_q/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2.3556414.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmpe6f5vo_q/model.ckpt-14184\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "你是谁 -> 我是你的小通\n",
      "你喜欢我吗 -> 我喜欢你\n",
      "给我唱一首歌 -> = =\n",
      "我帅吗 -> 你是最漂亮的男人\n"
     ]
    }
   ],
   "source": [
    "def greedy_decode(test_words, tf_estimator, dl):\n",
    "    test_indices = []\n",
    "    for test_word in test_words:\n",
    "        test_idx = [dl.source_word2idx[c] for c in test_word] + \\\n",
    "                   [dl.source_word2idx['<pad>']] * (args.source_max_len - len(test_word))\n",
    "        test_indices.append(test_idx)\n",
    "    test_indices = np.atleast_2d(test_indices)\n",
    "    \n",
    "    zeros = np.zeros([len(test_words), args.target_max_len], np.int64)\n",
    "\n",
    "    pred_ids = tf_estimator.predict(tf.estimator.inputs.numpy_input_fn(\n",
    "        x={'source':test_indices, 'target':zeros}, batch_size=len(test_words), shuffle=False))\n",
    "    pred_ids = list(pred_ids)\n",
    "    \n",
    "    target_idx2word = {i: w for w, i in dl.target_word2idx.items()}\n",
    "    for i, test_word in enumerate(test_words):\n",
    "        ans = ''.join([target_idx2word[id] for id in pred_ids[i]])\n",
    "        print(test_word, '->', ans.replace('<end>', ''))\n",
    "\n",
    "\n",
    "def prepare_params(dl):\n",
    "    if args.activation == 'relu':\n",
    "        activation = tf.nn.relu\n",
    "    elif args.activation == 'elu':\n",
    "        activation = tf.nn.elu\n",
    "    elif args.activation == 'lrelu':\n",
    "        activation = tf.nn.leaky_relu\n",
    "    else:\n",
    "        raise ValueError(\"acitivation fn has to be 'relu' or 'elu' or 'lrelu'\")\n",
    "    params = {\n",
    "        'source_vocab_size': len(dl.source_word2idx),\n",
    "        'target_vocab_size': len(dl.target_word2idx),\n",
    "        'start_symbol': dl.target_word2idx['<start>'],\n",
    "        'activation': activation}\n",
    "    return params\n",
    "\n",
    "\n",
    "def main():\n",
    "    dl = DataLoader(\n",
    "        source_path='../temp/dialog_source.txt',\n",
    "        target_path='../temp/dialog_target.txt')\n",
    "    sources, targets = dl.load()\n",
    "    print('Source Vocab Size:', len(dl.source_word2idx))\n",
    "    print('Target Vocab Size:', len(dl.target_word2idx))\n",
    "    \n",
    "    tf_estimator = tf.estimator.Estimator(\n",
    "        tf_estimator_model_fn, params=prepare_params(dl))\n",
    "    \n",
    "    for epoch in range(2):\n",
    "        tf_estimator.train(tf.estimator.inputs.numpy_input_fn(\n",
    "            x = {'source':sources, 'target':targets},\n",
    "            batch_size = args.batch_size,\n",
    "            shuffle = True))\n",
    "        greedy_decode(['你是谁', '你喜欢我吗', '给我唱一首歌', '我帅吗'], tf_estimator, dl)\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
